import json
import time
import jieba
from multiprocessing import Process, Queue, Manager


def getstoplist() -> set:
	"""
	从文件中读取并构建停用词表
	:return:停用词表（集合类型）
	"""
	stopdata = open(
		"D:/buaa_czw/2022 Autumn/General Programming 2022/Week 2/stopwords_list.txt",
		"r",
		encoding="utf-8"
	).readlines()
	stoplist = set()
	for line in stopdata:
		line = line.replace("\n", "")
		jieba.add_word(line)
		# 停用词表加入自定义词典中保证正确过滤停用词和统计词频
		stoplist.add(line)
	return stoplist


def data_load(data_path):
	contents = list()
	fp = open(data_path, 'r', encoding='utf-8')
	json_file = json.load(fp)
	for news in json_file:
		contents.append(news['content'])
	fp.close()
	return contents


def Map(text_list, word_list, stop_list):
	# word_list = list()
	for text in text_list:
		temp_text = jieba.lcut(text)
		for item in temp_text:
			if (item not in stop_list) and len(item) > 1:
				word_list.append(item)
	# queue.put(word_list)


def Reduce(word_list):
	word_dict = dict()
	# while True:
		# item = queue.get()
		# if item is None:
			# break
		# else:
			# for word in item:
				# word_dict[word] = word_dict.get(word, 0) + 1
	for word in word_list:
		word_dict[word] = word_dict.get(word, 0) + 1
	w_dict_order = sorted(word_dict.items(), key=lambda x: x[1], reverse=True)
	with open("dict_data.txt", 'w', encoding='utf-8') as f:
		for k, v in w_dict_order:
			f.write(f"单词 {k} 频数为：{v}.\n")


if __name__ == '__main__':
	total_number = 100000
	process_number = 100
	# q = Queue()
	news_cont = data_load(r'sohu_data.json')
	stop_words = getstoplist()
	# number_per_proc = round(len(news_cont)/process_number)
	number_per_proc = total_number//process_number
	# print(f"文件读取完毕...用时:{time.time() - start_time}")
	m = Manager()
	w_list = m.list()
	# w_list = list()
	process_list = list()
	for i in range(process_number):
		proc = Process(target=Map, args=(news_cont[i*number_per_proc: (i+1)*number_per_proc], w_list, stop_words))
		process_list.append(proc)
	start_time = time.time()
	print(f'进程创建完毕...当前时间:{start_time:.2f}')
	for p in process_list:
		p.start()
	for p in process_list:
		# print(f"等待{p.name}执行完毕...")
		p.join()
		# print(f"{p.name}执行完毕。")
	print(f"全部进程执行完毕...用时:{time.time() - start_time:.2f}")
	# q.put(None)
	Reduce(w_list)
	# print(f'词频写入文件成功...')
	print(f"程序执行完毕,进程数:{process_number},每个进程处理文本{number_per_proc}条,\n总用时{time.time() - start_time:.2f}")
